/*
 * Virtio PCI driver - modern (virtio 1.0) device support
 *
 * This module allows virtio devices to be used over a virtual PCI device.
 * This can be used with QEMU based VMMs like KVM or Xen.
 *
 * Copyright IBM Corp. 2007
 * Copyright Red Hat, Inc. 2014
 *
 * Authors:
 *  Anthony Liguori  <aliguori@us.ibm.com>
 *  Rusty Russell <rusty@rustcorp.com.au>
 *  Michael S. Tsirkin <mst@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 *
 */
#define  __SYLIXOS_KERNEL
#include <SylixOS.h>
#include <module.h>
#include <linux/compat.h>
#include <string.h>
#include <system/device/pci/pciBus.h>
#include <system/device/pci/pciMsi.h>
#include <system/device/pci/pciLib.h>
#include <system/device/pci/pciIds.h>
#include <system/device/pci/pciDev.h>
#include <system/device/pci/pciCap.h>
#include <system/device/pci/pciCapExt.h>

#define VIRTIO_PCI_NO_LEGACY
#include "virtio_pci_common.h"

ULONG API_VmmMap(PVOID pvVirtualAddr,
                PVOID pvPhysicalAddr,
                size_t stSize,
                ULONG ulFlag);

/*
 * These helpers provide future and backwards compatibility
 * for accessing popular PCI BAR info
 */
#define pci_resource_start(dev, bar)    ((dev)->PCIDEV_tResource[(bar)].PCIRS_stStart)
#define pci_resource_end(dev, bar)      ((dev)->PCIDEV_tResource[(bar)].PCIRS_stEnd)
#define pci_resource_flags(dev, bar)    ((dev)->PCIDEV_tResource[(bar)].PCIRS_ulFlags)
#define pci_resource_len(dev,bar) \
    ((pci_resource_end((dev), (bar)) == 0) ? 0 :    \
                            \
     (pci_resource_end((dev), (bar)) -      \
      pci_resource_start((dev), (bar)) + 1))

/*
 * Type-safe wrappers for io accesses.
 * Use these to enforce at compile time the following spec requirement:
 *
 * The driver MUST access each field using the 鈥渘atural鈥� access
 * method, i.e. 32-bit accesses for 32-bit fields, 16-bit accesses
 * for 16-bit fields and 8-bit accesses for 8-bit fields.
 */
static inline u8 vp_ioread8(u8 __iomem *addr, int type)
{
    if (type) {
        return read8((addr_t)addr);
    }

	return in8((addr_t)addr);
}
static inline u16 vp_ioread16 (__le16 __iomem *addr, int type)
{
    if (type) {
        return read16((addr_t)addr);
    }

    return in16((addr_t)addr);
}

static inline u32 vp_ioread32(__le32 __iomem *addr, int type)
{
    if (type) {
        return read32((addr_t)addr);
    }

    return in32((addr_t)addr);
}

static inline void vp_iowrite8(u8 value, u8 __iomem *addr, int type)
{
    if (type) {
        write8(value, (addr_t)addr);
    } else {
        out8(value, (addr_t)addr);
    }
}

static inline void vp_iowrite16(u16 value, __le16 __iomem *addr, int type)
{
    if (type) {
        write16(value, (addr_t)addr);
    } else {
        out16(value, (addr_t)addr);
    }
}

static inline void vp_iowrite32(u32 value, __le32 __iomem *addr, int type)
{
    if (type) {
        write32(value, (addr_t)addr);
    } else {
        out32(value, (addr_t)addr);
    }
}

static void vp_iowrite64_twopart(u64 val,
				 __le32 __iomem *lo, __le32 __iomem *hi, int type)
{
	vp_iowrite32((u32)val, lo, type);
	vp_iowrite32(val >> 32, hi, type);
}

static void __iomem *map_capability(PCI_DEV_HANDLE dev, int off,
				    size_t minlen,
				    u32 align,
				    u32 start, u32 size,
				    size_t *len)
{
	u8 bar;
	u32 offset, length;
	void __iomem *p = NULL;
	struct virtio_pci_device *vp_dev = dev->PCIDEV_pvPrivate;

	API_PciDevConfigReadByte(dev, off + offsetof(struct virtio_pci_cap, bar),
			     &bar);
	API_PciDevConfigReadDword(dev, off + offsetof(struct virtio_pci_cap, offset),
			     &offset);
	API_PciDevConfigReadDword(dev, off + offsetof(struct virtio_pci_cap, length),
			      &length);

	if (length <= start) {
	    _PrintFormat("virtio_pci: bad capability len %u (>%u expected)\r\n",
			length, start);
		return NULL;
	}

	if (length - start < minlen) {
	    _PrintFormat("virtio_pci: bad capability len %u (>=%zu expected)\r\n",
			length, minlen);
		return NULL;
	}

	length -= start;

	if (start + offset < offset) {
	    _PrintFormat("virtio_pci: map wrap-around %u+%u\r\n",
			start, offset);
		return NULL;
	}

	offset += start;

	if (offset & (align - 1)) {
	    _PrintFormat("virtio_pci: offset %u not aligned to %u\r\n",
			offset, align);
		return NULL;
	}

	if (length > size)
		length = size;

	if (len)
		*len = length;

	if (minlen + offset < minlen ||
	    minlen + offset > pci_resource_len(dev, bar)) {
	    _PrintFormat("virtio_pci: map virtio %u@%u "
			"out of range on bar %i length %lu\r\n",
			minlen, offset,
			bar, (unsigned long)pci_resource_len(dev, bar));
		return NULL;
	}

#ifdef SYLIXOS
    p = vp_dev->bar_vaddr[bar] + offset;
#else
	p = pci_iomap_range(dev, bar, offset, length);
	if (!p)
		printk("virtio_pci: unable to map virtio %u@%u on bar %d\n",
			length, offset, bar);
#endif

	return p;
}

static int pci_find_ext_capability(PCI_DEV_HANDLE pci_dev, int cap)
{
    u32 val = 0;

    API_PciCapExtFind (pci_dev->PCIDEV_iDevBus,
                       pci_dev->PCIDEV_iDevDevice,
                       pci_dev->PCIDEV_iDevFunction,
                       cap, &val);

    return val;
}

/* virtio config->get_features() implementation */
static u64 vp_get_features(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	u64 features;

	vp_iowrite32(0, &vp_dev->common->device_feature_select, vp_dev->common_type);
	features = vp_ioread32(&vp_dev->common->device_feature, vp_dev->common_type);
	vp_iowrite32(1, &vp_dev->common->device_feature_select, vp_dev->common_type);
	features |= ((u64)vp_ioread32(&vp_dev->common->device_feature, vp_dev->common_type) << 32);

	return features;
}

static void vp_transport_features(struct virtio_device *vdev, u64 features)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	PCI_DEV_HANDLE pci_dev = vp_dev->pci_dev;

	if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) &&
			pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV))
		__virtio_set_bit(vdev, VIRTIO_F_SR_IOV);
}

/* virtio config->finalize_features() implementation */
static int vp_finalize_features(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	u64 features = vdev->features;

	/* Give virtio_ring a chance to accept features. */
	vring_transport_features(vdev);

	/* Give virtio_pci a chance to accept features. */
	vp_transport_features(vdev, features);

	if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) {
	    _PrintFormat("virtio: device uses modern interface "
			"but does not have VIRTIO_F_VERSION_1\r\n");
		return -EINVAL;
	}

	vp_iowrite32(0, &vp_dev->common->guest_feature_select, vp_dev->common_type);
	vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature, vp_dev->common_type);
	vp_iowrite32(1, &vp_dev->common->guest_feature_select, vp_dev->common_type);
	vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature, vp_dev->common_type);

	return 0;
}

/* virtio config->get() implementation */
static void vp_get(struct virtio_device *vdev, unsigned offset,
		   void *buf, unsigned len)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	u8 b;
	__le16 w;
	__le32 l;

	BUG_ON(offset + len > vp_dev->device_len);

	switch (len) {
	case 1:
		b = vp_ioread8(vp_dev->device + offset, vp_dev->device_type);
		memcpy(buf, &b, sizeof b);
		break;
	case 2:
		w = cpu_to_le16(vp_ioread16(vp_dev->device + offset, vp_dev->device_type));
		memcpy(buf, &w, sizeof w);
		break;
	case 4:
		l = cpu_to_le32(vp_ioread32(vp_dev->device + offset, vp_dev->device_type));
		memcpy(buf, &l, sizeof l);
		break;
	case 8:
		l = cpu_to_le32(vp_ioread32(vp_dev->device + offset, vp_dev->device_type));
		memcpy(buf, &l, sizeof l);
		l = cpu_to_le32(vp_ioread32(vp_dev->device + offset + sizeof l, vp_dev->device_type));
		memcpy(buf + sizeof l, &l, sizeof l);
		break;
	default:
		BUG();
	}
}

/* the config->set() implementation.  it's symmetric to the config->get()
 * implementation */
static void vp_set(struct virtio_device *vdev, unsigned offset,
		   const void *buf, unsigned len)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	u8 b;
	__le16 w;
	__le32 l;

	BUG_ON(offset + len > vp_dev->device_len);

	switch (len) {
	case 1:
		memcpy(&b, buf, sizeof b);
		vp_iowrite8(b, vp_dev->device + offset, vp_dev->device_type);
		break;
	case 2:
		memcpy(&w, buf, sizeof w);
		vp_iowrite16(le16_to_cpu(w), vp_dev->device + offset, vp_dev->device_type);
		break;
	case 4:
		memcpy(&l, buf, sizeof l);
		vp_iowrite32(le32_to_cpu(l), vp_dev->device + offset, vp_dev->device_type);
		break;
	case 8:
		memcpy(&l, buf, sizeof l);
		vp_iowrite32(le32_to_cpu(l), vp_dev->device + offset, vp_dev->device_type);
		memcpy(&l, buf + sizeof l, sizeof l);
		vp_iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l, vp_dev->device_type);
		break;
	default:
		BUG();
	}
}

static u32 vp_generation(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	return vp_ioread8(&vp_dev->common->config_generation, vp_dev->common_type);
}

/* config->{get,set}_status() implementations */
static u8 vp_get_status(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	return vp_ioread8(&vp_dev->common->device_status, vp_dev->common_type);
}

static void vp_set_status(struct virtio_device *vdev, u8 status)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	/* We should never be setting status to 0. */
	BUG_ON(status == 0);
	vp_iowrite8(status, &vp_dev->common->device_status, vp_dev->common_type);
}

static void vp_reset(struct virtio_device *vdev)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	/* 0 status means a reset. */
	vp_iowrite8(0, &vp_dev->common->device_status, vp_dev->common_type);
	/* After writing 0 to device_status, the driver MUST wait for a read of
	 * device_status to return 0 before reinitializing the device.
	 * This will flush out the status write, and flush in device writes,
	 * including MSI-X interrupts, if any.
	 */
	while (vp_ioread8(&vp_dev->common->device_status, vp_dev->common_type))
		usleep(1000);
#ifndef SYLIXOS
	/* Flush pending VQ/configuration callbacks. */
	vp_synchronize_vectors(vdev);
#endif
}

static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
{
	/* Setup the vector used for configuration events */
	vp_iowrite16(vector, &vp_dev->common->msix_config, vp_dev->common_type);
	/* Verify we had enough resources to assign the vector */
	/* Will also flush the write out to device */
	return vp_ioread16(&vp_dev->common->msix_config, vp_dev->common_type);
}

static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev,
				  struct virtio_pci_vq_info *info,
				  unsigned index,
				  void (*callback)(struct virtqueue *vq),
				  const char *name,
				  bool ctx,
				  u16 msix_vec)
{
	struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common;
	struct virtqueue *vq;
	u16 num, off;

	if (index >= vp_ioread16(&cfg->num_queues, vp_dev->common_type))
		return NULL;

	/* Select the queue we're interested in */
	vp_iowrite16(index, &cfg->queue_select, vp_dev->common_type);

	/* Check if queue is either not available or already active. */
	num = vp_ioread16(&cfg->queue_size, vp_dev->common_type);
	if (!num || vp_ioread16(&cfg->queue_enable, vp_dev->common_type))
		return NULL;

	if (num & (num - 1)) {
	    _PrintFormat("bad queue size %u\r\n", num);
		return NULL;
	}

	/* get offset of notification word for this vq */
	off = vp_ioread16(&cfg->queue_notify_off, vp_dev->common_type);

	info->msix_vector = msix_vec;

	/* create the vring */
	vq = vring_create_virtqueue(index, num,
	        LW_CFG_CPU_ARCH_CACHE_LINE, &vp_dev->vdev,
				    true, true, ctx,
				    vp_notify, callback, name);
	if (!vq)
		return NULL;

	/* activate the queue */
	vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size, vp_dev->common_type);
	vp_iowrite64_twopart(virtqueue_get_desc_addr(vq),
			     &cfg->queue_desc_lo, &cfg->queue_desc_hi, vp_dev->common_type);
	vp_iowrite64_twopart(virtqueue_get_avail_addr(vq),
			     &cfg->queue_avail_lo, &cfg->queue_avail_hi, vp_dev->common_type);
	vp_iowrite64_twopart(virtqueue_get_used_addr(vq),
			     &cfg->queue_used_lo, &cfg->queue_used_hi, vp_dev->common_type);

	if (vp_dev->notify_base) {
		/* offset should not wrap */
		if ((u64)off * vp_dev->notify_offset_multiplier + 2
		    > vp_dev->notify_len) {
		    _PrintFormat("bad notification offset %u (x %u) "
				 "for queue %u > %zd\r\n",
				 off, vp_dev->notify_offset_multiplier,
				 index, vp_dev->notify_len);
			goto err_map_notify;
		}
		vq->priv = (void __force *)vp_dev->notify_base +
			off * vp_dev->notify_offset_multiplier;
		vq->priv_type = vp_dev->notify_type;
	} else {
		vq->priv = (void __force *)map_capability(vp_dev->pci_dev,
					  vp_dev->notify_map_cap, 2, 2,
					  off * vp_dev->notify_offset_multiplier, 2,
					  NULL);
		u8 bar;
        API_PciDevConfigReadByte(vp_dev->pci_dev, vp_dev->notify_map_cap + offsetof(struct virtio_pci_cap, bar), &bar);
        if (pci_resource_flags(vp_dev->pci_dev, bar) & PCI_IORESOURCE_IO) {
            vp_dev->notify_type = 0;
            vq->priv_type = vp_dev->notify_type;
        }
        if (pci_resource_flags(vp_dev->pci_dev, bar) & PCI_IORESOURCE_MEM) {
            vp_dev->notify_type = 1;
            vq->priv_type = vp_dev->notify_type;
        }
	}

	if (!vq->priv) {
		goto err_map_notify;
	}

	if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
		vp_iowrite16(msix_vec, &cfg->queue_msix_vector, vp_dev->common_type);
		msix_vec = vp_ioread16(&cfg->queue_msix_vector, vp_dev->common_type);
		if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
			goto err_assign_vector;
		}
	}

	return vq;

err_assign_vector:
#ifndef SYLIXOS
	if (!vp_dev->notify_base)
		pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv);
#endif
err_map_notify:
	vring_del_virtqueue(vq);
	return NULL;
}

static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs,
			      struct virtqueue *vqs[],
			      vq_callback_t *callbacks[],
			      const char * const names[], const bool *ctx,
			      void *desc)
{
	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
	struct virtqueue *vq;
	int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc);

	if (rc)
		return rc;

	/* Select and activate all queues. Has to be done last: once we do
	 * this, there's no way to go back except reset.
	 */
	list_for_each_entry(vq, &vdev->vqs, list) {
		vp_iowrite16(vq->index, &vp_dev->common->queue_select, vp_dev->common_type);
		vp_iowrite16(1, &vp_dev->common->queue_enable, vp_dev->common_type);
	}

	return 0;
}

static void del_vq(struct virtio_pci_vq_info *info)
{
	struct virtqueue *vq = info->vq;
	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);

	vp_iowrite16(vq->index, &vp_dev->common->queue_select, vp_dev->common_type);

	if (vp_dev->msix_enabled) {
		vp_iowrite16(VIRTIO_MSI_NO_VECTOR,
			     &vp_dev->common->queue_msix_vector, vp_dev->common_type);
		/* Flush the write out to device */
		vp_ioread16(&vp_dev->common->queue_msix_vector, vp_dev->common_type);
	}

#ifndef SYLIXOS
	if (!vp_dev->notify_base)
		pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv);
#endif

	vring_del_virtqueue(vq);
}

static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
	.get		= NULL,
	.set		= NULL,
	.generation	= vp_generation,
	.get_status	= vp_get_status,
	.set_status	= vp_set_status,
	.reset		= vp_reset,
	.find_vqs	= vp_modern_find_vqs,
	.del_vqs	= vp_del_vqs,
	.get_features	= vp_get_features,
	.finalize_features = vp_finalize_features,
	.bus_name	= vp_bus_name,
	.set_vq_affinity = vp_set_vq_affinity,
	.get_vq_affinity = vp_get_vq_affinity,
};

static const struct virtio_config_ops virtio_pci_config_ops = {
	.get		= vp_get,
	.set		= vp_set,
	.generation	= vp_generation,
	.get_status	= vp_get_status,
	.set_status	= vp_set_status,
	.reset		= vp_reset,
	.find_vqs	= vp_modern_find_vqs,
	.del_vqs	= vp_del_vqs,
	.get_features	= vp_get_features,
	.finalize_features = vp_finalize_features,
	.bus_name	= vp_bus_name,
	.set_vq_affinity = vp_set_vq_affinity,
	.get_vq_affinity = vp_get_vq_affinity,
};

// copy from linux
static u8 __pci_find_next_cap_ttl(int bus, int dev, int fn,
                                  u8 pos, int cap, int *ttl)
{
    u8 id;
    u16 ent;

    API_PciConfigInByte(bus, dev, fn, pos, &pos);

    while ((*ttl)--) {
        if (pos < 0x40)
            break;
        pos &= ~3;
        API_PciConfigInWord(bus, dev, fn, pos, &ent);

        id = ent & 0xff;
        if (id == 0xff)
            break;
        if (id == cap)
            return pos;
        pos = (ent >> 8);
    }
    return 0;
}

static u8 __pci_find_next_cap(int bus, int dev, int fn, u8 pos, int cap)
{
    int ttl = PCI_FIND_CAP_TTL;

    return __pci_find_next_cap_ttl(bus, dev, fn, pos, cap, &ttl);
}

static u8 pci_find_next_capability(PCI_DEV_HANDLE dev, u8 pos, int cap)
{
    return __pci_find_next_cap(dev->PCIDEV_iDevBus,
                                dev->PCIDEV_iDevDevice,
                                dev->PCIDEV_iDevFunction,
                                pos + PCI_CAP_LIST_NEXT, cap);
}

static u8 __pci_bus_find_cap_start(int bus, int dev, int fn, u8 hdr_type)
{
    u16 status;

    API_PciConfigInWord(bus, dev, fn, PCI_STATUS, &status);
    if (!(status & PCI_STATUS_CAP_LIST))
        return 0;

    switch (hdr_type) {
    case PCI_HEADER_TYPE_NORMAL:
    case PCI_HEADER_TYPE_BRIDGE:
        return PCI_CAPABILITY_LIST;
    case PCI_HEADER_TYPE_CARDBUS:
        return PCI_CB_CAPABILITY_LIST;
    }

    return 0;
}

static u8 pci_find_capability(PCI_DEV_HANDLE dev, int cap)
{
    u8 pos;

    pos = __pci_bus_find_cap_start(dev->PCIDEV_iDevBus,
                                   dev->PCIDEV_iDevDevice,
                                   dev->PCIDEV_iDevFunction,
                                   dev->PCIDEV_iType);
    if (pos)
        pos = __pci_find_next_cap(dev->PCIDEV_iDevBus,
                                  dev->PCIDEV_iDevDevice,
                                  dev->PCIDEV_iDevFunction,
                                  pos, cap);

    return pos;
}

/**
 * virtio_pci_find_capability - walk capabilities to find device info.
 * @dev: the pci device
 * @cfg_type: the VIRTIO_PCI_CAP_* value we seek
 * @ioresource_types: PCI_IORESOURCE_MEM and/or PCI_IORESOURCE_IO.
 *
 * Returns offset of the capability, or 0.
 */
static inline int virtio_pci_find_capability(PCI_DEV_HANDLE dev, u8 cfg_type,
					     u32 ioresource_types, int *bars)
{
	int pos;

	for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR);
	     pos > 0;
	     pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) {
		u8 type, bar;
		API_PciDevConfigReadByte(dev,
		                         pos + offsetof(struct virtio_pci_cap, cfg_type),
				                 &type);
		API_PciDevConfigReadByte(dev,
		                         pos + offsetof(struct virtio_pci_cap, bar),
				                 &bar);

		/* Ignore structures with reserved BAR values */
		if (bar > 0x5)
			continue;

		if (type == cfg_type) {
			if (pci_resource_len(dev, bar) &&
			    pci_resource_flags(dev, bar) & ioresource_types) {
				*bars |= (1 << bar);
				return pos;
			}
		}
	}
	return 0;
}

#ifndef SYLIXOS
/* This is part of the ABI.  Don't screw with it. */
static inline void check_offsets(void)
{
	/* Note: disk space was harmed in compilation of this function. */
	BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR !=
		     offsetof(struct virtio_pci_cap, cap_vndr));
	BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT !=
		     offsetof(struct virtio_pci_cap, cap_next));
	BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN !=
		     offsetof(struct virtio_pci_cap, cap_len));
	BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE !=
		     offsetof(struct virtio_pci_cap, cfg_type));
	BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR !=
		     offsetof(struct virtio_pci_cap, bar));
	BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET !=
		     offsetof(struct virtio_pci_cap, offset));
	BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH !=
		     offsetof(struct virtio_pci_cap, length));
	BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT !=
		     offsetof(struct virtio_pci_notify_cap,
			      notify_off_multiplier));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT !=
		     offsetof(struct virtio_pci_common_cfg,
			      device_feature_select));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF !=
		     offsetof(struct virtio_pci_common_cfg, device_feature));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT !=
		     offsetof(struct virtio_pci_common_cfg,
			      guest_feature_select));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF !=
		     offsetof(struct virtio_pci_common_cfg, guest_feature));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX !=
		     offsetof(struct virtio_pci_common_cfg, msix_config));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ !=
		     offsetof(struct virtio_pci_common_cfg, num_queues));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS !=
		     offsetof(struct virtio_pci_common_cfg, device_status));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION !=
		     offsetof(struct virtio_pci_common_cfg, config_generation));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT !=
		     offsetof(struct virtio_pci_common_cfg, queue_select));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE !=
		     offsetof(struct virtio_pci_common_cfg, queue_size));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX !=
		     offsetof(struct virtio_pci_common_cfg, queue_msix_vector));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE !=
		     offsetof(struct virtio_pci_common_cfg, queue_enable));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF !=
		     offsetof(struct virtio_pci_common_cfg, queue_notify_off));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO !=
		     offsetof(struct virtio_pci_common_cfg, queue_desc_lo));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI !=
		     offsetof(struct virtio_pci_common_cfg, queue_desc_hi));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO !=
		     offsetof(struct virtio_pci_common_cfg, queue_avail_lo));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI !=
		     offsetof(struct virtio_pci_common_cfg, queue_avail_hi));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO !=
		     offsetof(struct virtio_pci_common_cfg, queue_used_lo));
	BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI !=
		     offsetof(struct virtio_pci_common_cfg, queue_used_hi));
}
#endif

/* the PCI probing function */
int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev)
{
    PCI_DEV_HANDLE pci_dev = vp_dev->pci_dev;
	int err, common, isr, notify, device;
	u32 notify_length;
	u32 notify_offset;
    u8 bar;

#ifndef SYLIXOS
	check_offsets();
#endif

	/* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
	if (PCI_DEV_DEVICE_ID(pci_dev) < 0x1000 || PCI_DEV_DEVICE_ID(pci_dev) > 0x107f)
		return -ENODEV;

	if (PCI_DEV_DEVICE_ID(pci_dev) < 0x1040) {
		/* Transitional devices: use the PCI subsystem device id as
		 * virtio device id, same as legacy driver always did.
		 */
		vp_dev->vdev.id.device = PCI_DEV_SUB_SYSTEM_ID(pci_dev);
	} else {
		/* Modern devices: simply use PCI device id, but start from 0x1040. */
		vp_dev->vdev.id.device = PCI_DEV_DEVICE_ID(pci_dev) - 0x1040;
	}
	vp_dev->vdev.id.vendor = PCI_DEV_SUB_VENDOR_ID(pci_dev);

	/* check for a common config: if not, use legacy mode (bar 0). */
	common = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_COMMON_CFG,
					    PCI_IORESOURCE_IO | PCI_IORESOURCE_MEM,
					    &vp_dev->modern_bars);
	if (!common) {
	    _PrintFormat("virtio_pci: leaving for legacy driver\r\n");
		return -ENODEV;
	}

	/* If common is there, these should be too... */
	isr = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_ISR_CFG,
					 PCI_IORESOURCE_IO | PCI_IORESOURCE_MEM,
					 &vp_dev->modern_bars);
	notify = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG,
					    PCI_IORESOURCE_IO | PCI_IORESOURCE_MEM,
					    &vp_dev->modern_bars);
	if (!isr || !notify) {
	    _PrintFormat("virtio_pci: missing capabilities %d/%d/%d\r\n",
			common, isr, notify);
		return -EINVAL;
	}

#ifdef SYLIXOS
	/* check bar0 ~ bar5 type, iomap mmio bar. */
	int i;
	for (i = 0; i < 6; i++) {
	    /*
	     *  flag == 0 means this bar is not used.
	     *  io bar dont need iomap
	     */
	    if (!pci_resource_flags(pci_dev, i)) {
	        continue;
	    }

	    if (pci_resource_flags(pci_dev, i) & PCI_IORESOURCE_IO) {
	        vp_dev->bar_vaddr[i] = (void *)pci_resource_start(pci_dev, i);
        }

        if (pci_resource_flags(pci_dev, i) & PCI_IORESOURCE_MEM) {
            API_VmmMap((void *)pci_resource_start(pci_dev, i),
                       (void *)pci_resource_start(pci_dev, i),
                       pci_resource_len(pci_dev, i),
                       LW_VMM_FLAG_DMA);
            vp_dev->bar_vaddr[i] = (void *)pci_resource_start(pci_dev, i);
        }
	}
#else
	err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64));
	if (err)
		err = dma_set_mask_and_coherent(&pci_dev->dev,
						DMA_BIT_MASK(32));
	if (err)
		dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA.  Trying to continue, but this might not work.\n");
#endif

	/* Device capability is only mandatory for devices that have
	 * device-specific configuration.
	 */
	device = virtio_pci_find_capability(pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG,
					    PCI_IORESOURCE_IO | PCI_IORESOURCE_MEM,
					    &vp_dev->modern_bars);

#ifndef SYLIXOS
	err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars,
					   "virtio-pci-modern");
	if (err)
		return err;
#endif

	err = -EINVAL;
	vp_dev->common = map_capability(pci_dev, common,
					sizeof(struct virtio_pci_common_cfg), 4,
					0, sizeof(struct virtio_pci_common_cfg),
					NULL);
	if (!vp_dev->common)
		goto err_map_common;
    API_PciDevConfigReadByte(pci_dev, common + offsetof(struct virtio_pci_cap, bar), &bar);
    if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_IO) {
        vp_dev->common_type = 0;
    }
    if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_MEM) {
        vp_dev->common_type = 1;
    }

	vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1,
				     0, 1,
				     NULL);
	if (!vp_dev->isr)
		goto err_map_isr;
    API_PciDevConfigReadByte(pci_dev, isr + offsetof(struct virtio_pci_cap, bar), &bar);
    if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_IO) {
        vp_dev->isr_type = 0;
    }
    if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_MEM) {
        vp_dev->isr_type = 1;
    }

	/* Read notify_off_multiplier from config space. */
    API_PciDevConfigReadDword(pci_dev,
			      notify + offsetof(struct virtio_pci_notify_cap,
						notify_off_multiplier),
			      &vp_dev->notify_offset_multiplier);
	/* Read notify length and offset from config space. */
    API_PciDevConfigReadDword(pci_dev,
			      notify + offsetof(struct virtio_pci_notify_cap,
						cap.length),
			      &notify_length);

    API_PciDevConfigReadDword(pci_dev,
			      notify + offsetof(struct virtio_pci_notify_cap,
						cap.offset),
			      &notify_offset);

	/* We don't know how many VQs we'll map, ahead of the time.
	 * If notify length is small, map it all now.
	 * Otherwise, map each VQ individually later.
	 */
	if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) {
		vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2,
						     0, notify_length,
						     &vp_dev->notify_len);
		if (!vp_dev->notify_base)
			goto err_map_notify;
	    API_PciDevConfigReadByte(pci_dev, notify + offsetof(struct virtio_pci_cap, bar), &bar);
	    if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_IO) {
	        vp_dev->notify_type = 0;
	    }
	    if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_MEM) {
	        vp_dev->notify_type = 1;
	    }
	} else {
		vp_dev->notify_map_cap = notify;
	}

	/* Again, we don't know how much we should map, but PAGE_SIZE
	 * is more than enough for all existing devices.
	 */
	if (device) {
		vp_dev->device = map_capability(pci_dev, device, 0, 4,
						0, PAGE_SIZE,
						&vp_dev->device_len);
		if (!vp_dev->device)
			goto err_map_device;
        API_PciDevConfigReadByte(pci_dev, device + offsetof(struct virtio_pci_cap, bar), &bar);
        if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_IO) {
            vp_dev->device_type = 0;
        }
        if (pci_resource_flags(pci_dev, bar) & PCI_IORESOURCE_MEM) {
            vp_dev->device_type = 1;
        }
		vp_dev->vdev.config = &virtio_pci_config_ops;
	} else {
		vp_dev->vdev.config = &virtio_pci_config_nodev_ops;
	}

	snprintf(vp_dev->name, 32, "%s%s", virtio_device_type[vp_dev->vdev.id.device], "_pci_m");

	vp_dev->config_vector = vp_config_vector;
	vp_dev->setup_vq = setup_vq;
	vp_dev->del_vq = del_vq;

	return 0;

err_map_device:
err_map_notify:
err_map_isr:
err_map_common:
	return err;
}

void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev)
{
#ifndef SYLIXOS
	struct pci_dev *pci_dev = vp_dev->pci_dev;

	if (vp_dev->device)
		pci_iounmap(pci_dev, vp_dev->device);
	if (vp_dev->notify_base)
		pci_iounmap(pci_dev, vp_dev->notify_base);
	pci_iounmap(pci_dev, vp_dev->isr);
	pci_iounmap(pci_dev, vp_dev->common);
	pci_release_selected_regions(pci_dev, vp_dev->modern_bars);
#endif
}
